import csv
import sys
import os
import time

def exec_script(script_name, *args):
	cmd = str.format('node {0}', script_name)
	for arg in args:
		cmd += ' ' + arg
	fd = os.popen(cmd)
	output = fd.readlines()
	fd.close()
	return output

#0"Symbol"
#1"Name"
#2"LastSale"
#3"MarketCap"
#4"ADR TSO"
#5"IPOyear"
#6"Sector"
#7"industry"
#8"Summary Quote"

letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
indexes = range(len(letters))
tickers_processed = 0.0
tickers_partly_processed = 0.0
tickers_faild = 0.0
processed = []
with open('companies.csv', 'wb') as csvfile:
	company_writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
	for i in indexes:
		file_name = 'ndq_{0}.csv'.format(letters[i])
		f = open(file_name, 'rt')
		try:
			reader = csv.reader(f)
			skip = True
			for row in reader:
				if skip:
					skip = False
					continue
				try:
					tickers_processed += 1
					ticker = row[0].split('^')[0].upper()
					was_processed = [was_processed for was_processed in processed if was_processed == ticker]
					if len(was_processed) > 0:
						continue
					name = row[1]
					try:
						cap = float(row[3])
					except:
						cap = 0
					info = []
					counter = 0
					if cap == 0:
						while (counter < 5 and len(info) == 0):
							info = exec_script('yahoo_market_cap.js', ticker)
							counter += 1
						if len(info) != 0:
							cap = float(info[0])
					sector = row[6].strip()
					industry = row[7].strip()
					subIndustry = ''
					description = ''
					info = []
					counter = 0
					while (counter < 5 and len(info) == 0):
						info = exec_script('bloomberg_sector_info.js', ticker)
						counter += 1
					if len(info) == 4:
						sector = info[0].strip()
						industry = info[1].strip()
						subIndustry = info[2].strip()
						description = info[3].strip()
					else:
						tickers_partly_processed += 1
					company_writer.writerow([ticker, name, cap, sector, industry, subIndustry])
					csvfile.flush()
					partly_rate = (tickers_partly_processed / tickers_processed) * 100
					faild_rate = (tickers_faild / tickers_processed) * 100
					message = '{0} processed; {1} ({2:.2f}%) of them partly; {3} ({4:.2f}%) faild -----> {5}/${6} from {7}'
					message = message.format(tickers_processed, tickers_partly_processed, partly_rate, tickers_faild, faild_rate, ticker, cap, file_name)
					print message
				except:
					tickers_faild += 1
					partly_rate = (tickers_partly_processed / tickers_processed) * 100
					faild_rate = (tickers_faild / tickers_processed) * 100
					message = '{0} processed; {1} ({2:.2f}%) of them partly; {3} ({4:.2f}%) faild -----> {5}/${6} from {7}'
					message = message.format(tickers_processed, tickers_partly_processed, partly_rate, tickers_faild, faild_rate, ticker, 0.0, file_name)
		finally:
			f.close()
	csvfile.close()